[XEN] Implement faster int 0x80 handling for compat mode guests.

Using the GPF handler to spot the software interrupt and pass it back
to the guest increases the base syscall time by a factor of 2.7
compared with 32on32 using direct trap to ring 1. (0.3270->0.8680
microseconds, measured with lmbench lat_syscall).

Since the 64 bit IDT can only contain 64 bit segment selectors we
cannot trap directly to compat mode ring 1. However implementing a
dedicated 64 bit ring 0 trap handler allows us to avoid much of the
GPF handler overhead and reduces the overhead to 1.7 times
(0.3270->0.5497 microseconds).

Signed-off-by: Ian Campbell <ian.campbell@xensource.com>

diff --git a/xen/arch/x86/x86_64/asm-offsets.c b/xen/arch/x86/x86_64/asm-offsets.c
index 001a781235391852d5c68a1558121ac0373f0e25..93a047db3a9d0d4b8266bb6eaa1f2a46a2a229d9 100644 (file)
--- a/xen/arch/x86/x86_64/asm-offsets.c
+++ b/xen/arch/x86/x86_64/asm-offsets.c
@@ -59,6 +59,7 @@ void __dummy__(void)
     OFFSET(VCPU_domain, struct vcpu, domain);
     OFFSET(VCPU_vcpu_info, struct vcpu, vcpu_info);
     OFFSET(VCPU_trap_bounce, struct vcpu, arch.trap_bounce);
+    OFFSET(VCPU_int80_bounce, struct vcpu, arch.int80_bounce);
     OFFSET(VCPU_thread_flags, struct vcpu, arch.flags);
     OFFSET(VCPU_event_addr, struct vcpu,
            arch.guest_context.event_callback_eip);

diff --git a/xen/arch/x86/x86_64/compat/entry.S b/xen/arch/x86/x86_64/compat/entry.S
index 3f1d1b8d078f6da5a9b2d654a000b4fa7d4e918b..095bd61f73c24c4286d0ac222ef832d0227639ee 100644 (file)
--- a/xen/arch/x86/x86_64/compat/entry.S
+++ b/xen/arch/x86/x86_64/compat/entry.S
@@ -187,6 +187,10 @@ ENTRY(compat_post_handle_exception)
         call  compat_create_bounce_frame
         jmp   compat_test_all_events
 
+ENTRY(compat_int80_direct_trap)
+        call  compat_create_bounce_frame
+        jmp   compat_restore_all_guest
+
 /* CREATE A BASIC EXCEPTION FRAME ON GUEST OS (RING-1) STACK:            */
 /*   {[ERRCODE,] EIP, CS, EFLAGS, [ESP, SS]}                             */
 /* %rdx: trap_bounce, %rbx: struct vcpu                                  */

diff --git a/xen/arch/x86/x86_64/compat/traps.c b/xen/arch/x86/x86_64/compat/traps.c
index a7f24cecf88dfda6f63527ab099b2d6b0ec2289c..0172e846c4091c5294c74502adc2984bae86a765 100644 (file)
--- a/xen/arch/x86/x86_64/compat/traps.c
+++ b/xen/arch/x86/x86_64/compat/traps.c
@@ -1,6 +1,7 @@
 #ifdef CONFIG_COMPAT
 
 #include <xen/event.h>
+#include <asm/regs.h>
 #include <compat/callback.h>
 #include <compat/arch-x86_32.h>
 
@@ -291,6 +292,9 @@ int compat_set_trap_table(XEN_GUEST_HANDLE(trap_info_compat_t) traps)
 
         XLAT_trap_info(dst + cur.vector, &cur);
 
+        if ( cur.vector == 0x80 )
+            init_int80_direct_trap(current);
+
         guest_handle_add_offset(traps, 1);
     }
 

diff --git a/xen/arch/x86/x86_64/entry.S b/xen/arch/x86/x86_64/entry.S
index cffd89dcede0fd993ea364dd9232ac5459be5ee2..17311c724226d71c2a718cce974e8c26cf200454 100644 (file)
--- a/xen/arch/x86/x86_64/entry.S
+++ b/xen/arch/x86/x86_64/entry.S
@@ -222,6 +222,35 @@ bad_hypercall:
         movq $-ENOSYS,UREGS_rax(%rsp)
         jmp  test_all_events
 
+ENTRY(int80_direct_trap)
+        pushq $0
+        SAVE_ALL
+
+        GET_CURRENT(%rbx)
+
+        /* Check that the callback is non-null. */
+        leaq  VCPU_int80_bounce(%rbx),%rdx
+        cmp   $0, TRAPBOUNCE_flags(%rdx)
+        jz    int80_slow_path
+
+        movq  VCPU_domain(%rbx),%rax
+        btl   $_DOMF_compat,DOMAIN_domain_flags(%rax)
+        jc    compat_int80_direct_trap
+
+        call  create_bounce_frame
+        jmp   restore_all_guest
+
+int80_slow_path:
+        /* 
+         * Setup entry vector and error code as if this was a GPF caused by an
+         * IDT entry with DPL==0.
+         */
+        movl  $((0x80 << 3) | 0x2),UREGS_error_code(%rsp)
+        movl  $TRAP_gp_fault,UREGS_entry_vector(%rsp)
+        /* A GPF wouldn't have incremented the instruction pointer. */
+        sub   $2,UREGS_rip(%rsp)
+        jmp   handle_exception_saved
+
 /* CREATE A BASIC EXCEPTION FRAME ON GUEST OS STACK:                     */
 /*   { RCX, R11, [DS-GS,] [CR2,] [ERRCODE,] RIP, CS, RFLAGS, RSP, SS }   */
 /* %rdx: trap_bounce, %rbx: struct vcpu                           */
@@ -359,6 +388,7 @@ ENTRY(ret_from_intr)
 /* No special register assumptions. */
 ENTRY(handle_exception)
         SAVE_ALL
+handle_exception_saved:
         testb $X86_EFLAGS_IF>>8,UREGS_eflags+1(%rsp)
         jz    exception_with_ints_disabled
         sti

diff --git a/xen/arch/x86/x86_64/traps.c b/xen/arch/x86/x86_64/traps.c
index 0c6ee1de35402e8735a2b2714aab087bd818d422..7a255c291fcb3091533e00664bcdfef9c9071ddc 100644 (file)
--- a/xen/arch/x86/x86_64/traps.c
+++ b/xen/arch/x86/x86_64/traps.c
@@ -247,6 +247,7 @@ unsigned long do_iret(void)
 
 asmlinkage void syscall_enter(void);
 asmlinkage void compat_hypercall(void);
+asmlinkage void int80_direct_trap(void);
 void __init percpu_traps_init(void)
 {
     char *stack_bottom, *stack;
@@ -262,6 +263,7 @@ void __init percpu_traps_init(void)
 #ifdef CONFIG_COMPAT
         /* The hypercall entry vector is only accessible from ring 1. */
         _set_gate(idt_table+HYPERCALL_VECTOR, 15, 1, &compat_hypercall);
+        _set_gate(idt_table+0x80, 15, 3, &int80_direct_trap);
 #endif
     }
 
@@ -346,6 +348,22 @@ void __init percpu_traps_init(void)
     wrmsr(MSR_SYSCALL_MASK, EF_VM|EF_RF|EF_NT|EF_DF|EF_IE|EF_TF, 0U);
 }
 
+void init_int80_direct_trap(struct vcpu *v)
+{
+    struct trap_info *ti = &v->arch.guest_context.trap_ctxt[0x80];
+    struct trap_bounce *tb = &v->arch.int80_bounce;
+
+    if ( !guest_gate_selector_okay(v->domain, ti->cs) )
+         return;
+
+    tb->flags = TBF_EXCEPTION;
+    tb->cs    = ti->cs;
+    tb->eip   = ti->address;
+
+    if ( null_trap_bounce(v, tb) )
+        tb->flags = 0;
+}
+
 static long register_guest_callback(struct callback_register *reg)
 {
     long ret = 0;

diff --git a/xen/include/asm-x86/domain.h b/xen/include/asm-x86/domain.h
index 6c88e5abb0773bcfbfdbf1f6e0b7f7826199c9a1..bdc2f16f1254fbd766efd74dc1410d90b2f2804f 100644 (file)
--- a/xen/include/asm-x86/domain.h
+++ b/xen/include/asm-x86/domain.h
@@ -248,6 +248,9 @@ struct arch_vcpu
 #ifdef CONFIG_X86_32
     struct desc_struct int80_desc;
 #endif
+#ifdef CONFIG_X86_64
+    struct trap_bounce int80_bounce;
+#endif
 
     /* Virtual Machine Extensions */
     struct hvm_vcpu hvm_vcpu;

diff --git a/xen/include/asm-x86/processor.h b/xen/include/asm-x86/processor.h
index 321219c8539393894c9d45a5a2598dd93507fb1e..a878d86303c1b6cbab41b72c1297aa719e3af1a8 100644 (file)
--- a/xen/include/asm-x86/processor.h
+++ b/xen/include/asm-x86/processor.h
@@ -455,16 +455,16 @@ extern idt_entry_t *idt_tables[];
 
 extern struct tss_struct init_tss[NR_CPUS];
 
-#ifdef CONFIG_X86_32
-
 extern void init_int80_direct_trap(struct vcpu *v);
+
+#if defined(CONFIG_X86_32)
+
 #define set_int80_direct_trap(_ed)                  \
     (memcpy(idt_tables[(_ed)->processor] + 0x80,    \
             &((_ed)->arch.int80_desc), 8))
 
 #else
 
-#define init_int80_direct_trap(_ed) ((void)0)
 #define set_int80_direct_trap(_ed)  ((void)0)
 
 #endif